# -*- coding: utf-8 -*-
import scrapy
import db
import sys
import time
import json
import urllib,urllib2

def parse(content):
    jsonObj = json.loads(content)
    if(jsonObj['err'] !=0):
        return
    article = jsonObj['article']
    spider_id = article['id']
    format = article['format']
    content = article['content']
    image = article['image']
    image_size = json.dumps(article['image_size'])
    user = json.dumps(article['user'])
    if "null" == user :
        user = ""
    if "null" == image_size :
        image_size = ""

    if "video" == format:
        pic_url = article['pic_url']
        low_url = article['low_url']
        high_url = article['high_url']
    else:
        pic_url = "";
        low_url = "";
        high_url = "";

    db.insertQiubai(spider_id,format,content,image,image_size,pic_url,low_url,high_url,user);
if __name__ == "__main__":
    num = 5
    headers={"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36"}
    lastId = db.getLastId()
    endId = lastId+num
    db.addLastId(endId)
    preUrl = "http://m2.qiushibaike.com/article/"
    endUrl = "/comments?article=1"
    i = 0
    for i in range(1,num):
        curId = lastId+i
        url = preUrl+str(curId)+endUrl
        time.sleep(3);
        req = urllib2.Request(url,None,headers)
        res = urllib2.urlopen(req)
        content = res.read()
        parse(content)

        
